package com.zdb.demo.spark;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.scheduler.DAGScheduler;
import org.apache.spark.scheduler.TaskScheduler;
import scala.Tuple2;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class SparkCtxTest {

    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setMaster("local[4]").setAppName("test");
        JavaSparkContext jsc = new JavaSparkContext(conf);

        List<Long> list = new ArrayList<>();
        for (long i = 0L; i < 100L; i++) {
            list.add(i);
        }

        JavaPairRDD<Long, Long> cnt = jsc.parallelize(list).mapToPair((Long aLong) -> {
            return new Tuple2<Long, Long>(aLong % 10, aLong);
        }).aggregateByKey(0L, new Function2<Long, Long, Long>() {
            @Override public Long call(Long v1, Long v2) throws Exception {
                return v1 + v2;
            }
        }, new Function2<Long, Long, Long>() {
            @Override public Long call(Long v1, Long v2) throws Exception {
                return v1 + v2;
            }
        })      ;
        cnt.foreach((Tuple2<Long, Long> longLongTuple2) -> {
            System.out.println((longLongTuple2._1 + ":" + longLongTuple2._2));
        });

        for (int i = 0; i < 1000; i++) {
            try {
                DAGScheduler dagScheduler = jsc.sc().dagScheduler();
                TaskScheduler taskScheduler = dagScheduler.taskScheduler();
                TimeUnit.SECONDS.sleep(1000L);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
}
